package informationretrieval.extensions;

import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.en.EnglishPossessiveFilter;
import org.apache.lucene.analysis.en.PorterStemFilter;
import org.apache.lucene.analysis.miscellaneous.ASCIIFoldingFilter;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.util.Version;

public class StopwordAnalyzer extends StopwordAnalyzerBase {

	// Class variables
	private int maxTokenLength = 50;
	
	/**
	 * Default constructor
	 * @param version
	 * @param stopwords
	 */
	public StopwordAnalyzer(Version version, CharArraySet stopwords) {
		super(version, stopwords);
	}

	/**
	 * A method to handle stop words parsing
	 */
	@Override
	protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
//		Tokenizer source = new LowerCaseTokenizer(Version.LUCENE_48, reader);
//		return new TokenStreamComponents(source, new PorterStemFilter(source));
		
		final StandardTokenizer source = new StandardTokenizer(matchVersion, reader);
        source.setMaxTokenLength(maxTokenLength);

        TokenStream pipeline = source;
        pipeline = new StandardFilter(matchVersion, pipeline);
        pipeline = new EnglishPossessiveFilter(matchVersion, pipeline);
        pipeline = new ASCIIFoldingFilter(pipeline);
        pipeline = new LowerCaseFilter(matchVersion, pipeline);
        pipeline = new StopFilter(matchVersion, pipeline, stopwords);
        pipeline = new PorterStemFilter(pipeline);
        return new TokenStreamComponents(source, pipeline);
	}
}
